{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "import tifffile as tiff\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# img_2015 = cv2.imread('input/1_0_960_.jpg')\n",
    "\n",
    "# plt.figure(figsize=(18, 5))\n",
    "# plt.subplot(131)\n",
    "# plt.hist(np.ravel(img_2015[:,:,0]))\n",
    "# plt.subplot(132)\n",
    "# plt.hist(np.ravel(img_2015[:,:,1]))\n",
    "# plt.subplot(133)\n",
    "# plt.hist(np.ravel(img_2015[:,:,2]))\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 归一化\n",
    "def scale_percentile_n(matrix):\n",
    "    w, h, d = matrix.shape\n",
    "    matrix = np.reshape(matrix, [w * h, d]).astype(np.float64)\n",
    "    # Get 2nd and 98th percentile\n",
    "    mins = np.percentile(matrix, 1, axis=0)\n",
    "    maxs = np.percentile(matrix, 99, axis=0) - mins\n",
    "    matrix = (matrix - mins[None, :]) / maxs[None, :]\n",
    "    matrix = np.reshape(matrix, [w, h, d])\n",
    "    matrix = matrix.clip(0, 1)\n",
    "    return matrix      \n",
    "\n",
    "# 标准化\n",
    "# def z_score(x, axis=(0, 1)):\n",
    "#     x = np.array(x).astype(float)\n",
    "#     return (x-np.mean(x, axis=axis))/np.std(x, axis=axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3000, 15106, 4)\n"
     ]
    }
   ],
   "source": [
    "# FILE_2015 = '../preliminary/quickbird2015.tif'\n",
    "# FILE_2017 = '../preliminary/quickbird2017.tif'\n",
    "# FILE_2015 = '../preliminary/quickbird2015_preliminary_2.tif'\n",
    "# FILE_2017 = '../preliminary/quickbird2017_preliminary_2.tif'\n",
    "# img_2015 = tiff.imread(FILE_2015).transpose([1, 2, 0])\n",
    "# img_2017 = tiff.imread(FILE_2017).transpose([1, 2, 0])\n",
    "# print img_2015.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载final图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5106, 15106, 4)\n"
     ]
    }
   ],
   "source": [
    "FILE_2015 = '../preliminary/quickbird2015.tif'\n",
    "FILE_2017 = '../preliminary/quickbird2017.tif'\n",
    "img_2015 = tiff.imread(FILE_2015).transpose([1, 2, 0])\n",
    "img_2017 = tiff.imread(FILE_2017).transpose([1, 2, 0])\n",
    "print img_2015.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5106, 15106, 3)\n"
     ]
    }
   ],
   "source": [
    "# 设置通道数\n",
    "channel = 3\n",
    "img_2015 = img_2015[:,:,:channel]\n",
    "img_2017 = img_2017[:,:,:channel]\n",
    "print img_2015.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 加权平均3通道做灰度代替4通道"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5106, 15106, 5)\n"
     ]
    }
   ],
   "source": [
    "# img_2015 = img_2015.astype(float)\n",
    "# img_2017 = img_2017.astype(float)\n",
    "\n",
    "# temp_15 = np.ones((5106, 15106, 5))\n",
    "# temp_17 = np.ones((5106, 15106 ,5))\n",
    "\n",
    "# temp_15[:, :, :4] = img_2015[:, :, :4]\n",
    "# temp_17[:, :, :4] = img_2017[:, :, :4]\n",
    "# temp_15[:, :, 4] = np.dot(img_2015[:, :, :3], [0.299, 0.587, 0.114])\n",
    "# temp_17[:, :, 4] = np.dot(img_2017[:, :, :3], [0.299, 0.587, 0.114])\n",
    "\n",
    "# img_2015 = temp_15\n",
    "# img_2017 = temp_17\n",
    "\n",
    "# print(img_2015.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# img_size = 960\n",
    "\n",
    "# for i in range(im_2015.shape[0]//img_size + 1): # last 284\n",
    "#     for j in range(im_2015.shape[1]//img_size + 1): #last 2 too small, drop one\n",
    "#         im_name = str(i)+'_'+str(j)+'_'+str(img_size)+'_.jpg'\n",
    "#         tif_name = str(i)+'_'+str(j)+'_'+str(img_size)+'_.tif'\n",
    "#         cv2.imwrite(\"input/satellite/2017/\"+im_name,stretch_n(im_2017[i*img_size:i*img_size+img_size, j*img_size:j*img_size+img_size, :3])*255)\n",
    "#         cv2.imwrite(\"input/satellite/2015/\"+im_name,stretch_n(im_2015[i*img_size:i*img_size+img_size, j*img_size:j*img_size+img_size, :3])*255)\n",
    "#         tiff.imsave(\"input/satellite/2017/\"+tif_name,np.array(stretch_n(im_2017[i*img_size:i*img_size+img_size, j*img_size:j*img_size+img_size, :3])*255, dtype=np.uint8))\n",
    "#         tiff.imsave(\"input/satellite/2015/\"+tif_name,np.array(stretch_n(im_2015[i*img_size:i*img_size+img_size, j*img_size:j*img_size+img_size, :3])*255, dtype=np.uint8))\n",
    "#         cv2.imwrite(\"input/satellite/cada/\"+im_name,im_cada[i*img_size:i*img_size+img_size, j*img_size:j*img_size+img_size]*255)\n",
    "#         cv2.imwrite(\"input/satellite/tiny/\"+im_name,im_tiny[i*img_size:i*img_size+img_size, j*img_size:j*img_size+img_size]*255)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 将原图切割出960x960的图像并单独做归一化后保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation completed\n"
     ]
    }
   ],
   "source": [
    "#将图片切割成960x960\n",
    "IM_ROWS = 5106\n",
    "# IM_ROWS = 4000\n",
    "# IM_ROWS = 3000\n",
    "IM_COLS = 15106\n",
    "ROI_SIZE = 960\n",
    "\n",
    "DATA_DIR = 'input'\n",
    "\n",
    "if not os.path.exists(DATA_DIR + '/train960'):\n",
    "    os.makedirs(DATA_DIR + '/train960')\n",
    "\n",
    "for i in range(int(IM_ROWS // ROI_SIZE)+1):\n",
    "    for j in range(int(IM_COLS // ROI_SIZE)+1):\n",
    "        ss2 = '{}/train960/{}_{}_{}_{}.tif'.format(DATA_DIR, 2015, i, j, ROI_SIZE)\n",
    "        if os.path.exists(ss2):\n",
    "            continue\n",
    "        tiff.imsave(ss2, np.array(scale_percentile_n(img_2015[i*ROI_SIZE:(i+1)*ROI_SIZE,j*ROI_SIZE:(j+1)*ROI_SIZE])*255,dtype=np.uint8))\n",
    "#         tiff.imsave(ss2, z_score(img_2015[i*ROI_SIZE:(i+1)*ROI_SIZE,j*ROI_SIZE:(j+1)*ROI_SIZE]))\n",
    "    \n",
    "for i in range(int(IM_ROWS // ROI_SIZE)+1):\n",
    "    for j in range(int(IM_COLS // ROI_SIZE)+1):\n",
    "        ss2 = '{}/train960/{}_{}_{}_{}.tif'.format(DATA_DIR, 2017, i, j, ROI_SIZE)\n",
    "        if os.path.exists(ss2):\n",
    "            continue\n",
    "        tiff.imsave(ss2, np.array(scale_percentile_n(img_2017[i*ROI_SIZE:(i+1)*ROI_SIZE,j*ROI_SIZE:(j+1)*ROI_SIZE])*255,dtype=np.uint8))\n",
    "#         tiff.imsave(ss2, z_score(img_2017[i*ROI_SIZE:(i+1)*ROI_SIZE,j*ROI_SIZE:(j+1)*ROI_SIZE]))\n",
    "\n",
    "print('Generation completed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Generation completed! Create ', '7770', 'pictures')\n"
     ]
    }
   ],
   "source": [
    "# 生成训练用图像\n",
    "DATA_DIR = 'input'\n",
    "years = [2015, 2017]\n",
    "# years = [2015]\n",
    "cut_size = 160\n",
    "spacing = 120\n",
    "channel = 3 #[0,1,2]\n",
    "if not os.path.exists(DATA_DIR+'/train'):\n",
    "    os.makedirs(DATA_DIR+'/train')\n",
    "if not os.path.exists(DATA_DIR+'/label'):\n",
    "    os.makedirs(DATA_DIR+'/label')\n",
    "# if not os.path.exists(DATA_DIR+'/labeltxt'):\n",
    "#     os.makedirs(DATA_DIR+'/labeltxt')\n",
    "count = 0\n",
    "flag = 0\n",
    "if cut_size%spacing != 0:\n",
    "    flag = 1\n",
    "for year in years:\n",
    "    for i in range(int(IM_ROWS // ROI_SIZE + 1)):\n",
    "        for j in range(int(IM_COLS // ROI_SIZE + 1)):\n",
    "            ss2 = '{}/train960/{}_{}_{}_{}.tif'.format(DATA_DIR, year, i, j, ROI_SIZE)\n",
    "            ss2_label = '{}/mylabel_{}/{}_{}_{}_.jpg'.format(DATA_DIR, year, i, j, ROI_SIZE)\n",
    "            if os.path.exists(ss2) and os.path.exists(ss2_label):\n",
    "                img = tiff.imread(ss2)\n",
    "                img_label = cv2.imread(ss2_label, -1)\n",
    "                for x in range(int(ROI_SIZE // spacing - (cut_size//spacing + flag - 1))):\n",
    "                    for y in range(int(ROI_SIZE // spacing - (cut_size//spacing + flag - 1))):\n",
    "                        ss3 = '{}/train/{}_{}_{}_{}_{}.tif'.format(DATA_DIR, year, i, j, x, y)\n",
    "                        ss3_label = '{}/label/{}_{}_{}_{}_{}.jpg'.format(DATA_DIR, year, i, j, x, y)\n",
    "                        ss3_txt = '{}/labeltxt/{}_{}_{}_{}_{}.txt'.format(DATA_DIR, year, i, j, x, y)\n",
    "                        img_temp = img[x*spacing:(x*spacing+cut_size), y*spacing:(y*spacing+cut_size)]\n",
    "                        if img_temp.shape != (cut_size, cut_size, channel):\n",
    "                            continue\n",
    "                        tiff.imsave(ss3, img_temp)\n",
    "                        cv2.imwrite(ss3_label,img_label[x*spacing:(x*spacing+cut_size), y*spacing:(y*spacing+cut_size)])\n",
    "#                         np.savetxt(ss3_txt, img_label[x*spacing:(x*spacing+cut_size),y*spacing:(y*spacing+cut_size),0] > 128, fmt='%d')\n",
    "                        count += 1\n",
    "#                         image = tiff.imread(ss3)\n",
    "#                         if image is None:\n",
    "#                             os.remove(ss3)\n",
    "#                         else:\n",
    "#                             if (image.shape[0]==cut_size) and (image.shape[1]==cut_size):\n",
    "#                                 count += 1\n",
    "#                                 cv2.imwrite(ss3_label,img_label[x*spacing:(x*spacing+cut_size), y*spacing:(y*spacing+cut_size)])\n",
    "#                                 np.savetxt(ss3_txt,img_label[x*spacing:(x*spacing+cut_size),y*spacing:(y*spacing+cut_size),0] > 128, fmt='%d')\n",
    "#                             else:\n",
    "#                                 os.remove(ss3)\n",
    "\n",
    "print('Generation completed! Create ', str(count), 'pictures')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "103217710447\n",
      "86692180482\n",
      "Generation completed\n"
     ]
    }
   ],
   "source": [
    "# 切割测试用图像（已弃用，整图切成小图然后进行归一化操作，效果不好）\n",
    "import tifffile as tiff\n",
    "import os\n",
    "import cv2\n",
    "\n",
    "IM_ROWS = 5106\n",
    "IM_COLS = 15106\n",
    "ROI_SIZE = 160\n",
    "\n",
    "DATA_DIR = 'input'\n",
    "if not os.path.exists(DATA_DIR+'/result_image'):\n",
    "    os.makedirs(DATA_DIR+'/result_image')\n",
    "years = [2015, 2017]\n",
    "for year in years:\n",
    "    img = eval('img_{}'.format(year))\n",
    "    print(img.sum())\n",
    "    for i in range(int(IM_ROWS // ROI_SIZE)): \n",
    "        for j in range(int(IM_COLS // ROI_SIZE)): \n",
    "            ss2 = '{}/result_image/{}_{}_{}_{}.tif'.format(DATA_DIR, year, i, j, ROI_SIZE)\n",
    "            if os.path.exists(ss2):\n",
    "                continue\n",
    "            tiff.imsave(ss2, scale_percentile_n(img[i*ROI_SIZE:(i+1)*ROI_SIZE, j*ROI_SIZE:(j+1)*ROI_SIZE]))\n",
    "\n",
    "    # 把下面i+1行的边角图片向上扩增成ROI_SIZE的图片\n",
    "    if IM_ROWS%ROI_SIZE != 0:\n",
    "        i = i+1\n",
    "        for k in range(int(IM_COLS // ROI_SIZE)):\n",
    "            ss2 = '{}/result_image/{}_{}_{}_{}.tif'.format(DATA_DIR, year, i, k, ROI_SIZE)\n",
    "            if os.path.exists(ss2):\n",
    "                continue\n",
    "            tiff.imsave(ss2, scale_percentile_n(img[(IM_ROWS-ROI_SIZE):IM_ROWS, k*ROI_SIZE:(k+1)*ROI_SIZE]))\n",
    "    # 把右面j+1列的边角图片向左扩增成ROI_SIZE的图片\n",
    "    if IM_COLS%ROI_SIZE != 0:\n",
    "        j = j+1\n",
    "        for k in range(int(IM_ROWS // ROI_SIZE)):\n",
    "            ss2 = '{}/result_image/{}_{}_{}_{}.tif'.format(DATA_DIR, year, k, j, ROI_SIZE)\n",
    "            if os.path.exists(ss2):\n",
    "                continue\n",
    "            tiff.imsave(ss2, scale_percentile_n(img[k*ROI_SIZE:(k+1)*ROI_SIZE, (IM_COLS-ROI_SIZE):IM_COLS]))\n",
    "    # 把右下角的图片扩增成ROI_SIZE的图片\n",
    "    if IM_ROWS%ROI_SIZE != 0 and IM_ROWS%ROI_SIZE != 0:\n",
    "        ss2 = '{}/result_image/{}_{}_{}_{}.tif'.format(DATA_DIR, year, i, j, ROI_SIZE)\n",
    "        if os.path.exists(ss2):\n",
    "            continue\n",
    "        tiff.imsave(ss2, scale_percentile_n(img[(IM_ROWS-ROI_SIZE):IM_ROWS, (IM_COLS-ROI_SIZE):IM_COLS]))\n",
    "    \n",
    "print('Generation completed')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
